<!DOCTYPE html>
<html lang="zh-CN">
<head>
  <meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=2">
<meta name="theme-color" content="#222">
<meta name="generator" content="Hexo 6.3.0">
  <link rel="apple-touch-icon" sizes="180x180" href="/images/apple-touch-icon-next.png">
  <link rel="icon" type="image/png" sizes="32x32" href="/images/favicon.jpg">
  <link rel="icon" type="image/png" sizes="16x16" href="/images/favicon.jpg">
  <link rel="mask-icon" href="/images/logo.svg" color="#222">

<link rel="stylesheet" href="/css/main.css">


<link rel="stylesheet" href="/lib/font-awesome/css/all.min.css">

<script id="hexo-configurations">
    var NexT = window.NexT || {};
    var CONFIG = {"hostname":"matian.connorma.cn","root":"/","scheme":"Muse","version":"7.8.0","exturl":false,"sidebar":{"position":"left","display":"post","padding":18,"offset":12,"onmobile":false},"copycode":{"enable":false,"show_result":false,"style":null},"back2top":{"enable":true,"sidebar":false,"scrollpercent":false},"bookmark":{"enable":false,"color":"#222","save":"auto"},"fancybox":false,"mediumzoom":false,"lazyload":false,"pangu":false,"comments":{"style":"tabs","active":null,"storage":true,"lazyload":false,"nav":null},"algolia":{"hits":{"per_page":10},"labels":{"input_placeholder":"Search for Posts","hits_empty":"We didn't find any results for the search: ${query}","hits_stats":"${hits} results found in ${time} ms"}},"localsearch":{"enable":false,"trigger":"auto","top_n_per_article":1,"unescape":false,"preload":false},"motion":{"enable":true,"async":false,"transition":{"post_block":"fadeIn","post_header":"slideDownIn","post_body":"slideDownIn","coll_header":"slideLeftIn","sidebar":"slideUpIn"}}};
  </script>

  <meta name="description" content="在之前特意穿插了一篇介绍bitmap的笔记bitmap与布隆过滤器，这里将介绍一个bitmap的应用场景，为了避免可能的风险，将具体的业务替换成了类似的场景。 需要开发一个图片标签功能：  用户发布图片时，可以为图片选择预置标签或新增标签；已发布的图片可以增加&#x2F;移除标签；一张图片的标签不重复，标签数量限制不能完全确定但是不会很多（十个以内） 系统预置标签主要是常用的、较为标准化的，如二次">
<meta property="og:type" content="article">
<meta property="og:title" content="bitmap的应用—内容标签场景的存储方案">
<meta property="og:url" content="http://matian.connorma.cn/4978e888.html">
<meta property="og:site_name" content="马天">
<meta property="og:description" content="在之前特意穿插了一篇介绍bitmap的笔记bitmap与布隆过滤器，这里将介绍一个bitmap的应用场景，为了避免可能的风险，将具体的业务替换成了类似的场景。 需要开发一个图片标签功能：  用户发布图片时，可以为图片选择预置标签或新增标签；已发布的图片可以增加&#x2F;移除标签；一张图片的标签不重复，标签数量限制不能完全确定但是不会很多（十个以内） 系统预置标签主要是常用的、较为标准化的，如二次">
<meta property="og:locale" content="zh_CN">
<meta property="article:published_time" content="2023-02-07T10:22:23.000Z">
<meta property="article:modified_time" content="2024-04-26T11:48:37.396Z">
<meta property="article:author" content="马天">
<meta property="article:tag" content="bitmap">
<meta name="twitter:card" content="summary">

<link rel="canonical" href="http://matian.connorma.cn/4978e888.html">


<script id="page-configurations">
  // https://hexo.io/docs/variables.html
  CONFIG.page = {
    sidebar: "",
    isHome : false,
    isPost : true,
    lang   : 'zh-CN'
  };
</script>

  <title>bitmap的应用—内容标签场景的存储方案 | 马天</title>
  






  <noscript>
  <style>
  .use-motion .brand,
  .use-motion .menu-item,
  .sidebar-inner,
  .use-motion .post-block,
  .use-motion .pagination,
  .use-motion .comments,
  .use-motion .post-header,
  .use-motion .post-body,
  .use-motion .collection-header { opacity: initial; }

  .use-motion .site-title,
  .use-motion .site-subtitle {
    opacity: initial;
    top: initial;
  }

  .use-motion .logo-line-before i { left: initial; }
  .use-motion .logo-line-after i { right: initial; }
  </style>
</noscript>

</head>

<body itemscope itemtype="http://schema.org/WebPage">
  <div class="container use-motion">
    <div class="headband"></div>

    <header class="header" itemscope itemtype="http://schema.org/WPHeader">
      <div class="header-inner"><div class="site-brand-container">
  <div class="site-nav-toggle">
    <div class="toggle" aria-label="切换导航栏">
      <span class="toggle-line toggle-line-first"></span>
      <span class="toggle-line toggle-line-middle"></span>
      <span class="toggle-line toggle-line-last"></span>
    </div>
  </div>

  <div class="site-meta">

    <a href="/" class="brand" rel="start">
      <span class="logo-line-before"><i></i></span>
      <h1 class="site-title">马天</h1>
      <span class="logo-line-after"><i></i></span>
    </a>
  </div>

  <div class="site-nav-right">
    <div class="toggle popup-trigger">
    </div>
  </div>
</div>




<nav class="site-nav">
  <ul id="menu" class="main-menu menu">
        <li class="menu-item menu-item-home">

    <a href="/" rel="section"><i class="fa fa-home fa-fw"></i>首页</a>

  </li>
        <li class="menu-item menu-item-about">

    <a href="/about/" rel="section"><i class="fa fa-user fa-fw"></i>关于</a>

  </li>
        <li class="menu-item menu-item-tags">

    <a href="/tags/" rel="section"><i class="fa fa-tags fa-fw"></i>标签</a>

  </li>
        <li class="menu-item menu-item-categories">

    <a href="/categories/" rel="section"><i class="fa fa-th fa-fw"></i>分类</a>

  </li>
        <li class="menu-item menu-item-archives">

    <a href="/archives/" rel="section"><i class="fa fa-archive fa-fw"></i>归档</a>

  </li>
  </ul>
</nav>




</div>
    </header>

    
  <div class="back-to-top">
    <i class="fa fa-arrow-up"></i>
    <span>0%</span>
  </div>


    <main class="main">
      <div class="main-inner">
        <div class="content-wrap">
          

          <div class="content post posts-expand">
            

    
  
  
  <article itemscope itemtype="http://schema.org/Article" class="post-block" lang="zh-CN">
    <link itemprop="mainEntityOfPage" href="http://matian.connorma.cn/4978e888.html">

    <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
      <meta itemprop="image" content="/images/avatar.gif">
      <meta itemprop="name" content="马天">
      <meta itemprop="description" content="程序猿">
    </span>

    <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
      <meta itemprop="name" content="马天">
    </span>
      <header class="post-header">
        <h1 class="post-title" itemprop="name headline">
          bitmap的应用—内容标签场景的存储方案
        </h1>

        <div class="post-meta">
            <span class="post-meta-item">
              <span class="post-meta-item-icon">
                <i class="far fa-calendar"></i>
              </span>
              <span class="post-meta-item-text">发表于</span>

              <time title="创建时间：2023-02-07 18:22:23" itemprop="dateCreated datePublished" datetime="2023-02-07T18:22:23+08:00">2023-02-07</time>
            </span>
            <span class="post-meta-item">
              <span class="post-meta-item-icon">
                <i class="far fa-folder"></i>
              </span>
              <span class="post-meta-item-text">分类于</span>
                <span itemprop="about" itemscope itemtype="http://schema.org/Thing">
                  <a href="/categories/%E5%9C%BA%E6%99%AF%E7%A4%BA%E4%BE%8B/" itemprop="url" rel="index"><span itemprop="name">场景示例</span></a>
                </span>
            </span>

          

        </div>
      </header>

    
    
    
    <div class="post-body" itemprop="articleBody">

      
        <p>在之前特意穿插了一篇介绍bitmap的笔记<a href="/660efd9a.html" title="bitmap与布隆过滤器">bitmap与布隆过滤器</a>，这里将介绍一个bitmap的应用场景，为了避免可能的风险，将具体的业务替换成了类似的场景。</p>
<p>需要开发一个图片标签功能：</p>
<ul>
<li>用户发布图片时，可以为图片选择预置标签或新增标签；已发布的图片可以增加&#x2F;移除标签；一张图片的标签不重复，标签数量限制不能完全确定但是不会很多（十个以内）</li>
<li>系统预置标签主要是常用的、较为标准化的，如二次元、水墨画、动漫风等。系统预置标签的文本可以修改，比如 动漫–&gt;动漫风</li>
<li>对用户新增的标签，不能再修改或删除标签内容；标签任意人可以使用，标签文本一致即认为相同</li>
<li>可以按单个标签或者多个标签联合使用（交、并、差）来筛选图片</li>
</ul>
<p>针对这一场景，设想了以下的几种方案，一一来进行说明</p>
<span id="more"></span>

<h2 id="数据库单列存储标签数组字符串"><a href="#数据库单列存储标签数组字符串" class="headerlink" title="数据库单列存储标签数组字符串"></a>数据库单列存储标签数组字符串</h2><p>增加表t_tags存储所有标签，内容如：</p>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre></td><td class="code"><pre><span class="line">1  动漫</span><br><span class="line">2  日漫</span><br><span class="line">...</span><br><span class="line">1000  机器猫</span><br><span class="line">.....</span><br></pre></td></tr></table></figure>

<p>在存储图片内容的表中增加一列tags，存储标签ID列表（使用逗号分割），如：1,2,1000</p>
<p>如此在查询图片内容时可以同时带出标签列表（对tags列分割为多行（利用substring_index）后join；或者查询出tags内容反序列化后再查t_tags表）；但是：</p>
<ul>
<li>难以使用标签搜索图片，like “%tagid%”的方式比较低效；</li>
<li>修改图片的标签列表时，必须对tags列反序列化修改后再序列化存储；</li>
<li>交、并、差等操作难以进行</li>
</ul>
<h2 id="关联表存储图片-标签多对多关系"><a href="#关联表存储图片-标签多对多关系" class="headerlink" title="关联表存储图片-标签多对多关系"></a>关联表存储图片-标签多对多关系</h2><p>图片和标签是一个多对多的关系，处理多对多的情况那么学院派的做法可以引入一个关联表t_img_tags，使用以下sql创建示例表：</p>
<figure class="highlight sql"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment">-- 图片存储表</span></span><br><span class="line"><span class="keyword">CREATE</span> <span class="keyword">TABLE</span> `t_imgs` (</span><br><span class="line">  `id` <span class="type">bigint</span>(<span class="number">20</span>) <span class="keyword">NOT</span> <span class="keyword">NULL</span>,</span><br><span class="line">  `img_name` <span class="type">char</span>(<span class="number">16</span>) <span class="type">CHARACTER</span> <span class="keyword">SET</span> utf8mb4 <span class="keyword">COLLATE</span> utf8mb4_bin <span class="keyword">NOT</span> <span class="keyword">NULL</span>,</span><br><span class="line">  <span class="keyword">PRIMARY</span> KEY (`id`)</span><br><span class="line">) ENGINE<span class="operator">=</span>InnoDB <span class="keyword">DEFAULT</span> CHARSET<span class="operator">=</span>utf8mb4 <span class="keyword">COLLATE</span><span class="operator">=</span>utf8mb4_bin COMMENT<span class="operator">=</span><span class="string">&#x27;图片存储表&#x27;</span>;</span><br><span class="line"></span><br><span class="line"><span class="keyword">INSERT</span> <span class="keyword">INTO</span> `t_imgs` (`id`, `img_name`) <span class="keyword">VALUES</span> (<span class="number">1</span>, <span class="string">&#x27;a.png&#x27;</span>);</span><br><span class="line"><span class="keyword">INSERT</span> <span class="keyword">INTO</span> `t_imgs` (`id`, `img_name`) <span class="keyword">VALUES</span> (<span class="number">2</span>, <span class="string">&#x27;b.jpg&#x27;</span>);</span><br><span class="line"><span class="keyword">INSERT</span> <span class="keyword">INTO</span> `t_imgs` (`id`, `img_name`) <span class="keyword">VALUES</span> (<span class="number">3</span>, <span class="string">&#x27;c.jpeg&#x27;</span>);</span><br><span class="line"></span><br><span class="line"><span class="comment">-- 标签记录表</span></span><br><span class="line"><span class="keyword">CREATE</span> <span class="keyword">TABLE</span> `t_tags` (</span><br><span class="line">  `id` <span class="type">bigint</span>(<span class="number">20</span>) <span class="keyword">NOT</span> <span class="keyword">NULL</span>,</span><br><span class="line">  `tag` <span class="type">varchar</span>(<span class="number">32</span>) <span class="keyword">COLLATE</span> utf8mb4_bin <span class="keyword">NOT</span> <span class="keyword">NULL</span>,</span><br><span class="line">  <span class="keyword">PRIMARY</span> KEY (`id`),</span><br><span class="line">  <span class="keyword">UNIQUE</span> KEY `tag` (`tag`)</span><br><span class="line">) ENGINE<span class="operator">=</span>InnoDB <span class="keyword">DEFAULT</span> CHARSET<span class="operator">=</span>utf8mb4 <span class="keyword">COLLATE</span><span class="operator">=</span>utf8mb4_bin COMMENT<span class="operator">=</span><span class="string">&#x27;标签记录表&#x27;</span>;</span><br><span class="line"></span><br><span class="line"><span class="keyword">INSERT</span> <span class="keyword">INTO</span> `t_tags` (`id`, `tag`) <span class="keyword">VALUES</span> (<span class="number">1</span>, <span class="string">&#x27;动漫&#x27;</span>);</span><br><span class="line"><span class="keyword">INSERT</span> <span class="keyword">INTO</span> `t_tags` (`id`, `tag`) <span class="keyword">VALUES</span> (<span class="number">2</span>, <span class="string">&#x27;日漫&#x27;</span>);</span><br><span class="line"><span class="keyword">INSERT</span> <span class="keyword">INTO</span> `t_tags` (`id`, `tag`) <span class="keyword">VALUES</span> (<span class="number">1001</span>, <span class="string">&#x27;机器猫&#x27;</span>);</span><br><span class="line"></span><br><span class="line"><span class="comment">-- 图片-标签关联表</span></span><br><span class="line"><span class="keyword">CREATE</span> <span class="keyword">TABLE</span> `t_img_tag` (</span><br><span class="line">  `img_id` <span class="type">bigint</span>(<span class="number">20</span>) <span class="keyword">NOT</span> <span class="keyword">NULL</span>,</span><br><span class="line">  `tag_id` <span class="type">bigint</span>(<span class="number">20</span>) <span class="keyword">NOT</span> <span class="keyword">NULL</span>,</span><br><span class="line">  <span class="keyword">PRIMARY</span> KEY (`img_id`,`tag_id`)</span><br><span class="line">) ENGINE<span class="operator">=</span>InnoDB <span class="keyword">DEFAULT</span> CHARSET<span class="operator">=</span>utf8mb4 <span class="keyword">COLLATE</span><span class="operator">=</span>utf8mb4_bin COMMENT<span class="operator">=</span><span class="string">&#x27;图片-标签关联表&#x27;</span>;</span><br><span class="line"></span><br><span class="line"><span class="keyword">INSERT</span> <span class="keyword">INTO</span> `t_img_tag` (`img_id`, `tag_id`) <span class="keyword">VALUES</span> (<span class="number">1</span>, <span class="number">1001</span>);</span><br><span class="line"><span class="keyword">INSERT</span> <span class="keyword">INTO</span> `t_img_tag` (`img_id`, `tag_id`) <span class="keyword">VALUES</span> (<span class="number">1</span>, <span class="number">2</span>);</span><br><span class="line"><span class="keyword">INSERT</span> <span class="keyword">INTO</span> `t_img_tag` (`img_id`, `tag_id`) <span class="keyword">VALUES</span> (<span class="number">2</span>, <span class="number">1</span>);</span><br><span class="line"><span class="keyword">INSERT</span> <span class="keyword">INTO</span> `t_img_tag` (`img_id`, `tag_id`) <span class="keyword">VALUES</span> (<span class="number">3</span>, <span class="number">1001</span>);</span><br><span class="line"></span><br><span class="line"><span class="keyword">ALTER</span> <span class="keyword">TABLE</span> `test`.`t_img_tag` </span><br><span class="line"><span class="keyword">ADD</span> <span class="keyword">FOREIGN</span> KEY (`img_id`) <span class="keyword">REFERENCES</span> `test`.`t_imgs` (`id`),</span><br><span class="line"><span class="keyword">ADD</span> <span class="keyword">FOREIGN</span> KEY (`tag_id`) <span class="keyword">REFERENCES</span> `test`.`t_tags` (`id`);</span><br></pre></td></tr></table></figure>

<p>此时查询图片的标签可以直接通过关联表join，或者分多步查询来取得最终结果；根据单个标签筛选图片时也是如此</p>
<p>多个标签AND时可以通过关联表的join进行</p>
<figure class="highlight sql"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment">-- 查询同时存在id为2、1001的标签的所有图片的id</span></span><br><span class="line"><span class="keyword">SELECT</span></span><br><span class="line">	t1.img_id </span><br><span class="line"><span class="keyword">FROM</span></span><br><span class="line">	( <span class="keyword">SELECT</span> <span class="operator">*</span> <span class="keyword">FROM</span> t_img_tag <span class="keyword">WHERE</span> tag_id <span class="operator">=</span> <span class="number">2</span> ) t1</span><br><span class="line">	<span class="keyword">INNER</span> <span class="keyword">JOIN</span> ( <span class="keyword">SELECT</span> <span class="operator">*</span> <span class="keyword">FROM</span> t_img_tag <span class="keyword">WHERE</span> tag_id <span class="operator">=</span> <span class="number">1001</span> ) t2 <span class="keyword">ON</span> t1.img_id <span class="operator">=</span> t2.img_id</span><br></pre></td></tr></table></figure>

<p>标签的差集也是类似的思路：</p>
<figure class="highlight sql"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment">-- 查询存在标签1、不存在标签1001的所有图片的id</span></span><br><span class="line"><span class="keyword">SELECT</span> <span class="keyword">DISTINCT</span></span><br><span class="line">	t1.img_id </span><br><span class="line"><span class="keyword">FROM</span></span><br><span class="line">	t_img_tag t1</span><br><span class="line">	<span class="keyword">INNER</span> <span class="keyword">JOIN</span> ( <span class="keyword">SELECT</span> <span class="operator">*</span> <span class="keyword">FROM</span> t_img_tag <span class="keyword">WHERE</span> img_id <span class="keyword">NOT</span> <span class="keyword">IN</span> ( <span class="keyword">SELECT</span> img_id <span class="keyword">FROM</span> t_img_tag <span class="keyword">WHERE</span> tag_id <span class="operator">=</span> <span class="number">1001</span> ) ) t2 <span class="keyword">ON</span> t1.img_id <span class="operator">=</span> t2.img_id </span><br><span class="line"><span class="keyword">WHERE</span></span><br><span class="line">	t1.tag_id <span class="operator">=</span> <span class="number">1</span></span><br></pre></td></tr></table></figure>

<p>引入关联表后，对标签的更新、查询（交、差集等）的问题得到一定的解决；当然，考虑到绝大多数查询会是查询图片的标签，那么也可以在t_imgs表保留tags字段，以图片更新标签&#x2F;标签文本更新时的额外操作来换取查询时的便捷</p>
<p>这种方式其实足以应对功能需求，但是在数据量大时表现如何？联合查询的情况下索引效率如何？恐怕很难说</p>
<h2 id="nosql存储图片-标签关系"><a href="#nosql存储图片-标签关系" class="headerlink" title="nosql存储图片-标签关系"></a>nosql存储图片-标签关系</h2><p>以文档型nosql mongodb为例，可以在img的document中添加一个array类型的字段来存储图片关联的标签，在这一个array字段建立索引以优化查询</p>
<blockquote>
<p>示例后补</p>
</blockquote>
<p>引入mongo需要考虑以下问题。在我们的系统中，图片存储记录属于重要度高的数据，因其敏感性，不可能只依赖mongo存储（事务、数据可靠性的保障），因此考虑图片及标签使用mysql存储，在标签更新时同步更新mongo记录，通过标签查询时使用mongo；如果可以接受数据延迟的情况，也可以通过MQ等异步处理，或者监听mysql binlog异步写入，甚至是通过离线任务定时同步数据。</p>
<p>在此之上就不得不继续考虑引入mongo的必要性：是否有必要引入mongo？引入一个中间件带来的运维部署成本，对比引入mongo的需求，是否有绝对的必要呢？</p>
<p>考虑mongo的优势，后续如果涉及图片、标签数据的分析处理，或者可能涉及到mongo擅长的gis地理位置信息，那么引入mongo也无可厚非；如果还可能涉及到大量数据的检索，那么可以考虑ES</p>
<h2 id="bitmap"><a href="#bitmap" class="headerlink" title="bitmap"></a>bitmap</h2><p>bitmap适合于高效标记大量数据的有无，并且可高效的进行交、并、补集运算，这简直是为bitmap量身定制的需求呀：</p>
<ul>
<li>为每个标签维护一个bitmap，记录图片的标签使用状态</li>
<li>假设有百万规模的图片，每个标签为了记录所有图片的关联状态，需要有百万bit的空间，按照 2^20 bit &#x3D; 1048576 bit 计算，每个标签占用128KB</li>
<li>在某些标签关联的图片很少即数据稀疏时，是不是浪费了部分内存？使用<a href="/660efd9a.html" title="bitmap与布隆过滤器">bitmap与布隆过滤器</a>介绍的RoaringBitmap等实现，可以在数据稀疏时减少内存浪费。如果量真的达到了不得不治理的程度（标签很多），相信进行数据分析、找出“僵尸标签”的成本也不难接受了</li>
<li>考虑到实际系统中id通常使用雪花ID，为了高效利用bitmap，可以将img_id减去固定的初始偏移值后再存储到bitmap，在读&#x2F;写时进行处理（可以封装bitmap类，类似装饰类的思路）</li>
<li>bitmap本身就极适于集合的交、并、差等计算，用作多标签的交、并、差的筛选很适合</li>
</ul>
<p>接下来是另一个问题：bitmap的数据如何存储？redis支持bitmap（setbit、getbit、bitcount、bitop等指令），但是如何持久化呢？</p>
<p>思考如何持久化，首先得想清楚在mysql中如何存储bitmap。在mysql 8中测试，mysql支持bit类型，但是最大长度为64；binary类型存储二进制数据，但是最大长度为255；varbinary是binary的可变长类型，最大程度可以达到mysql的单行最大长度65535（有其他列时需减去其他列占用的长度），与期望的能容纳百万bit差距甚远；</p>
<p>因此，只能考虑blob&#x2F;text类型，但text类型作为文本类型，即使选用ASCII编码，单个’1’也将被视为字符’1’占用一字节。因此只能考虑blob类型，blob类型最大可达到64KB，MEDIUMBLOB类型可以达到16MB，而更大的LONGBLOB可以达到4GB。当然实际的单次能操作的数据量，还需要考虑网络包大小、mysql支持的客户端传输的单个包体积限制等</p>
<p>为了足够存储2^20规模图片的状态，需要128KB的空间，我们可以使用MEDIUMBLOB；当然，也可以联用两个blob的列来拼出128KB的空间，但是如此将不能直接在这一列上执行二进制操作</p>
<p>这里使用mediumblob类型进行演示：</p>
<figure class="highlight sql"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">CREATE</span> <span class="keyword">TABLE</span> `t_label_bitmap` (</span><br><span class="line">  `label_id` <span class="type">bigint</span>(<span class="number">20</span>) <span class="keyword">NOT</span> <span class="keyword">NULL</span>,</span><br><span class="line">  `img_bitmap` mediumblob,</span><br><span class="line">  <span class="keyword">PRIMARY</span> KEY (`label_id`)</span><br><span class="line">) ENGINE<span class="operator">=</span>InnoDB <span class="keyword">DEFAULT</span> CHARSET<span class="operator">=</span>utf8mb4 <span class="keyword">COLLATE</span><span class="operator">=</span>utf8mb4_bin;</span><br><span class="line"></span><br><span class="line"><span class="comment">-- demo数据</span></span><br><span class="line"><span class="keyword">INSERT</span> <span class="keyword">INTO</span> `t_label_bitmap` (`label_id`, `img_bitmap`) <span class="keyword">VALUES</span> (<span class="number">1</span>, <span class="keyword">NULL</span>);</span><br></pre></td></tr></table></figure>

<p>使用jdk bitset + JDBC原生API操作示例如下：</p>
<figure class="highlight java"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment">// 1048576 = 2^20</span></span><br><span class="line"><span class="type">BitSet</span> <span class="variable">bitSet</span> <span class="operator">=</span> <span class="keyword">new</span> <span class="title class_">BitSet</span>(<span class="number">1048576</span>);</span><br><span class="line">bitSet.set(<span class="number">12</span>);</span><br><span class="line">bitSet.set(<span class="number">12344</span>);</span><br><span class="line"></span><br><span class="line">System.out.println(bitSet.get(<span class="number">12</span>));</span><br><span class="line">System.out.println(bitSet.get(<span class="number">12344</span>));</span><br><span class="line">System.out.println(bitSet.toString());</span><br><span class="line"></span><br><span class="line"><span class="type">Connection</span> <span class="variable">connection</span> <span class="operator">=</span> DriverManager.getConnection(</span><br><span class="line">        <span class="string">&quot;jdbc:mysql://localhost:3306/test?useSSL=false&amp;useUnicode=true&amp;characterEncoding=UTF-8&quot;</span>,</span><br><span class="line">        <span class="string">&quot;root&quot;</span>, <span class="string">&quot;root.123&quot;</span>);</span><br><span class="line"><span class="type">PreparedStatement</span> <span class="variable">statement</span> <span class="operator">=</span> connection.prepareStatement(</span><br><span class="line">        <span class="string">&quot;update t_label_bitmap set img_bitmap = ? where label_id = ?&quot;</span>);</span><br><span class="line">statement.setBytes(<span class="number">1</span>, bitSet.toByteArray());</span><br><span class="line">statement.setLong(<span class="number">2</span>, <span class="number">1</span>);</span><br><span class="line"><span class="type">boolean</span> <span class="variable">b</span> <span class="operator">=</span> statement.execute();</span><br><span class="line"></span><br><span class="line">statement.clearParameters();</span><br><span class="line">System.out.println();</span><br><span class="line"></span><br><span class="line">statement = connection.prepareStatement(<span class="string">&quot;select label_id,img_bitmap from t_label_bitmap where label_id = ?&quot;</span>);</span><br><span class="line">statement.setLong(<span class="number">1</span>, <span class="number">1</span>);</span><br><span class="line"><span class="type">ResultSet</span> <span class="variable">resultSet</span> <span class="operator">=</span> statement.executeQuery();</span><br><span class="line"><span class="keyword">while</span> (resultSet.next()) &#123;</span><br><span class="line">    <span class="type">long</span> <span class="variable">id</span> <span class="operator">=</span> resultSet.getLong(<span class="number">1</span>);</span><br><span class="line">    <span class="type">byte</span>[] bitmap = resultSet.getBytes(<span class="number">2</span>);</span><br><span class="line">    bitSet = BitSet.valueOf(bitmap);</span><br><span class="line">&#125;</span><br><span class="line">System.out.println(bitSet.get(<span class="number">12</span>));</span><br><span class="line">System.out.println(bitSet.get(<span class="number">12344</span>));</span><br><span class="line">System.out.println(bitSet.toString());</span><br><span class="line"></span><br><span class="line">resultSet.close();</span><br><span class="line">statement.close();</span><br><span class="line">connection.close();</span><br></pre></td></tr></table></figure>

<p>在执行update后可以看到，img_bitmap列存储了1.51KB的数据（因为bitset的实际空间是按需分配的），同样的可以根据byte数组还原bitset</p>
<p>也可以使用spring-data-jdbc提供的JdbcTemplate类完成同样的操作：</p>
<figure class="highlight java"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment">// 1048576 = 2^20</span></span><br><span class="line"><span class="type">BitSet</span> <span class="variable">bitSet</span> <span class="operator">=</span> <span class="keyword">new</span> <span class="title class_">BitSet</span>(<span class="number">1048576</span>);</span><br><span class="line">bitSet.set(<span class="number">12</span>);</span><br><span class="line">bitSet.set(<span class="number">12344</span>);</span><br><span class="line"></span><br><span class="line">System.out.println(bitSet.get(<span class="number">12</span>));</span><br><span class="line">System.out.println(bitSet.get(<span class="number">12344</span>));</span><br><span class="line">System.out.println(bitSet.toString());</span><br><span class="line"></span><br><span class="line"><span class="type">BitSet</span> <span class="variable">finalBitSet</span> <span class="operator">=</span> bitSet;</span><br><span class="line">jdbcTemplate.update(<span class="string">&quot;update t_label_bitmap set img_bitmap = ? where label_id = ?&quot;</span>,</span><br><span class="line">        <span class="keyword">new</span> <span class="title class_">PreparedStatementSetter</span>() &#123;</span><br><span class="line">            <span class="meta">@Override</span></span><br><span class="line">            <span class="keyword">public</span> <span class="keyword">void</span> <span class="title function_">setValues</span><span class="params">(PreparedStatement ps)</span> <span class="keyword">throws</span> SQLException &#123;</span><br><span class="line">                ps.setBytes(<span class="number">1</span>, finalBitSet.toByteArray());</span><br><span class="line">                ps.setLong(<span class="number">2</span>, <span class="number">1</span>);</span><br><span class="line">            &#125;</span><br><span class="line">        &#125;);</span><br><span class="line"></span><br><span class="line">System.out.println();</span><br><span class="line"></span><br><span class="line">bitSet = jdbcTemplate.query(<span class="string">&quot;select label_id,img_bitmap from t_label_bitmap where label_id = ?&quot;</span>,</span><br><span class="line">        <span class="keyword">new</span> <span class="title class_">PreparedStatementSetter</span>() &#123;</span><br><span class="line">            <span class="meta">@Override</span></span><br><span class="line">            <span class="keyword">public</span> <span class="keyword">void</span> <span class="title function_">setValues</span><span class="params">(PreparedStatement ps)</span> <span class="keyword">throws</span> SQLException &#123;</span><br><span class="line">                ps.setLong(<span class="number">1</span>, <span class="number">1</span>);</span><br><span class="line">            &#125;</span><br><span class="line">        &#125;,</span><br><span class="line">        <span class="keyword">new</span> <span class="title class_">ResultSetExtractor</span>&lt;BitSet&gt;() &#123;</span><br><span class="line">            <span class="meta">@Override</span></span><br><span class="line">            <span class="keyword">public</span> BitSet <span class="title function_">extractData</span><span class="params">(ResultSet rs)</span> <span class="keyword">throws</span> SQLException, DataAccessException &#123;</span><br><span class="line">                <span class="keyword">while</span> (rs.next()) &#123;</span><br><span class="line">                    <span class="keyword">return</span> BitSet.valueOf(rs.getBytes(<span class="number">2</span>));</span><br><span class="line">                &#125;</span><br><span class="line">                <span class="keyword">return</span> <span class="literal">null</span>;</span><br><span class="line">            &#125;</span><br><span class="line">        &#125;);</span><br><span class="line"></span><br><span class="line">System.out.println(bitSet.get(<span class="number">12</span>));</span><br><span class="line">System.out.println(bitSet.get(<span class="number">12344</span>));</span><br><span class="line">System.out.println(bitSet.toString());</span><br></pre></td></tr></table></figure>

<h3 id="bitmap如何持久化"><a href="#bitmap如何持久化" class="headerlink" title="bitmap如何持久化"></a>bitmap如何持久化</h3><p>以上的实例中我们解决了bitmap持久化的读写问题，但是在实际业务中怎样来做持久化操作是最合适的呢？在读写效率和数据可靠性上如何选择呢？</p>
<p>直接使用mysql存储bitmap，不引入redis，可以实现但是效率堪忧；使用redis作为mysql的缓存，查询可以得到加速，但是在频繁更新的时候需要大量的序列化、反序列化以及写大对象的操作。因此，如果标签数据不是很敏感，更推荐redis bitmap + 定时持久化到mysql的方式；或者，能够接受有人del key 或 flush 操作后操作数据恢复的话，完全依赖redis的持久化机制也并非不可行</p>

    </div>

    
    
    

      <footer class="post-footer">
          <div class="post-tags">
              <a href="/tags/bitmap/" rel="tag"># bitmap</a>
          </div>

        


        
    <div class="post-nav">
      <div class="post-nav-item"></div>
      <div class="post-nav-item">
    <a href="/660efd9a.html" rel="next" title="bitmap与布隆过滤器">
      bitmap与布隆过滤器 <i class="fa fa-chevron-right"></i>
    </a></div>
    </div>
      </footer>
    
  </article>
  
  
  



          </div>
          

<script>
  window.addEventListener('tabs:register', () => {
    let { activeClass } = CONFIG.comments;
    if (CONFIG.comments.storage) {
      activeClass = localStorage.getItem('comments_active') || activeClass;
    }
    if (activeClass) {
      let activeTab = document.querySelector(`a[href="#comment-${activeClass}"]`);
      if (activeTab) {
        activeTab.click();
      }
    }
  });
  if (CONFIG.comments.storage) {
    window.addEventListener('tabs:click', event => {
      if (!event.target.matches('.tabs-comment .tab-content .tab-pane')) return;
      let commentClass = event.target.classList[1];
      localStorage.setItem('comments_active', commentClass);
    });
  }
</script>

        </div>
          
  
  <div class="toggle sidebar-toggle">
    <span class="toggle-line toggle-line-first"></span>
    <span class="toggle-line toggle-line-middle"></span>
    <span class="toggle-line toggle-line-last"></span>
  </div>

  <aside class="sidebar">
    <div class="sidebar-inner">

      <ul class="sidebar-nav motion-element">
        <li class="sidebar-nav-toc">
          文章目录
        </li>
        <li class="sidebar-nav-overview">
          站点概览
        </li>
      </ul>

      <!--noindex-->
      <div class="post-toc-wrap sidebar-panel">
          <div class="post-toc motion-element"><ol class="nav"><li class="nav-item nav-level-2"><a class="nav-link" href="#%E6%95%B0%E6%8D%AE%E5%BA%93%E5%8D%95%E5%88%97%E5%AD%98%E5%82%A8%E6%A0%87%E7%AD%BE%E6%95%B0%E7%BB%84%E5%AD%97%E7%AC%A6%E4%B8%B2"><span class="nav-number">1.</span> <span class="nav-text">数据库单列存储标签数组字符串</span></a></li><li class="nav-item nav-level-2"><a class="nav-link" href="#%E5%85%B3%E8%81%94%E8%A1%A8%E5%AD%98%E5%82%A8%E5%9B%BE%E7%89%87-%E6%A0%87%E7%AD%BE%E5%A4%9A%E5%AF%B9%E5%A4%9A%E5%85%B3%E7%B3%BB"><span class="nav-number">2.</span> <span class="nav-text">关联表存储图片-标签多对多关系</span></a></li><li class="nav-item nav-level-2"><a class="nav-link" href="#nosql%E5%AD%98%E5%82%A8%E5%9B%BE%E7%89%87-%E6%A0%87%E7%AD%BE%E5%85%B3%E7%B3%BB"><span class="nav-number">3.</span> <span class="nav-text">nosql存储图片-标签关系</span></a></li><li class="nav-item nav-level-2"><a class="nav-link" href="#bitmap"><span class="nav-number">4.</span> <span class="nav-text">bitmap</span></a><ol class="nav-child"><li class="nav-item nav-level-3"><a class="nav-link" href="#bitmap%E5%A6%82%E4%BD%95%E6%8C%81%E4%B9%85%E5%8C%96"><span class="nav-number">4.1.</span> <span class="nav-text">bitmap如何持久化</span></a></li></ol></li></ol></div>
      </div>
      <!--/noindex-->

      <div class="site-overview-wrap sidebar-panel">
        <div class="site-author motion-element" itemprop="author" itemscope itemtype="http://schema.org/Person">
  <p class="site-author-name" itemprop="name">马天</p>
  <div class="site-description" itemprop="description">程序猿</div>
</div>
<div class="site-state-wrap motion-element">
  <nav class="site-state">
      <div class="site-state-item site-state-posts">
          <a href="/archives/">
        
          <span class="site-state-item-count">41</span>
          <span class="site-state-item-name">日志</span>
        </a>
      </div>
      <div class="site-state-item site-state-categories">
            <a href="/categories/">
          
        <span class="site-state-item-count">17</span>
        <span class="site-state-item-name">分类</span></a>
      </div>
      <div class="site-state-item site-state-tags">
            <a href="/tags/">
          
        <span class="site-state-item-count">25</span>
        <span class="site-state-item-name">标签</span></a>
      </div>
  </nav>
</div>
  <div class="links-of-author motion-element">
      <span class="links-of-author-item">
        <a href="https://github.com/matian2014" title="GitHub → https:&#x2F;&#x2F;github.com&#x2F;matian2014" rel="noopener" target="_blank"><i class="fab fa-github fa-fw"></i>GitHub</a>
      </span>
      <span class="links-of-author-item">
        <a href="mailto:matian1103@qq.com" title="E-Mail → mailto:matian1103@qq.com" rel="noopener" target="_blank"><i class="fa fa-envelope fa-fw"></i>E-Mail</a>
      </span>
  </div>



      </div>

    </div>
  </aside>
  <div id="sidebar-dimmer"></div>


      </div>
    </main>

    <footer class="footer">
      <div class="footer-inner">
        

        
  <div class="beian"><a href="https://beian.miit.gov.cn/" rel="noopener" target="_blank">陕ICP备2023001563号 </a>
  </div>

<div class="copyright">
  
  &copy; 
  <span itemprop="copyrightYear">2025</span>
  <span class="with-love">
    <i class="fa fa-heart"></i>
  </span>
  <span class="author" itemprop="copyrightHolder">马天</span>
</div>
  <div class="powered-by">由 <a href="https://hexo.io/" class="theme-link" rel="noopener" target="_blank">Hexo</a> & <a href="https://muse.theme-next.org/" class="theme-link" rel="noopener" target="_blank">NexT.Muse</a> 强力驱动
  </div>

        








      </div>
    </footer>
  </div>

  
  <script src="/lib/anime.min.js"></script>
  <script src="/lib/velocity/velocity.min.js"></script>
  <script src="/lib/velocity/velocity.ui.min.js"></script>

<script src="/js/utils.js"></script>

<script src="/js/motion.js"></script>


<script src="/js/schemes/muse.js"></script>


<script src="/js/next-boot.js"></script>




  















  

  

</body>
</html>
